In [2]:
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
from scipy.optimize import minimize
import warnings
warnings.filterwarnings('ignore')
from sklearn.preprocessing import MinMaxScaler
In [3]:
df = pd.read_csv('./nasdaq_d_2012_2022.csv')
df = df.pivot(index="date",columns="tic", values="adjcp")
df.index = pd.to_datetime(df.index, format = '%Y-%m-%d')
tick = ['TSLA', 'NVDA', 'AMD', 'AVGO', 'ALGN', 'LRCX', 'FTNT', 
          'ODFL', 'AMAT','MSFT', 'MU', 'IDXX', 'KLAC', 'CDNS', 
          'MRVL', 'ASML', 'FB', 'CTAS','INTU', 'SNPS']
df = df.loc[:,tick]
In [ ]:
#TODO:NOISE 随机扰动
class portOptim(object):
    def __init__(self,real_df,pred_df, dynamic_win = True):
        self.df = real_df.iloc[1:,:]
        self.date = pred_df.index
        self.pct_dt = real_df.pct_change().iloc[1:,:]
        self.pred_df = pred_df
        self.day = 0
        self.max_day = pred_df.shape[0]
        self.covWin = 30
        self.data = self.df.iloc[self.day:self.day+self.covWin,:]
        self.init_action = self.action = np.array([0]*20)
        self.mu = 0
        self.cov = 0
        self.risk_re = 0.5
        self.dynamic_win = dynamic_win
        
        #portfolio
        self.p = lambda A: self.asset + np.append(-A.sum()-abs(A).sum()*self.tran_cost,A)

        #weight
        self.w = lambda A: (self.p(A) / np.sum(self.p(A)))

        #Penalties for handling fees
        self.pena = lambda A: abs(A).sum()/self.p(A).sum()
        
        #sharpe ratio
        self.SR = lambda A: (self.mu@self.w(A)-self.pena(A))/np.sqrt(self.w(A)@self.cov@self.w(A))
        self.op = lambda A: -self.SR(A)


        ones = np.ones(21)
        self.cons = ({'type': 'ineq', 'fun': lambda A: self.w(A)},
                {'type': 'eq', 'fun': lambda A: self.w(A) @ ones - 1})
        
        self.tran_cost = 0.001
        self.rf = 0.06/365
        self.asset = self.weight = np.array([1/21]*21)
        self.scaler = MinMaxScaler(feature_range=(-30,-5))
        self.action_memory = []
        self.asset_memory = []
        self.tick = np.array(tick)
        print(f'Day{self.day}, initial portfolio {self.asset} , total {round(self.asset.sum(),2)}')
        
    
    def get_mu(self):
        cov = np.array(self.data.cov())
        var = cov.diagonal().reshape(20,1)
        win_s = self.scaler.fit_transform(var).astype('int').flatten()
        pred_c = np.array(self.pred_df.iloc[self.day,:]/self.df.iloc[self.day+self.covWin-1,:])
        mu = []
        for i in range(len(win_s)):
            pct_c = self.data.iloc[win_s[i]:,i]
            mu.append(pct_c.mean())

        dynamic_mu = np.append(self.rf,np.array(mu*pred_c-1)/(win_s+1))
        normal_mu = np.append(self.rf,np.array(self.data.mean()+pred_c/self.covWin))

        print(f'Var\n{var.squeeze()}\n winSize\n {win_s}\n mu \n{dynamic_mu}')

        return dynamic_mu if self.dynamic_win else normal_mu
        
    def optim(self):
        result = minimize(self.op, x0=(self.action), method='trust-constr', constraints=self.cons)
        return result.x
                      
    def step(self):
        if self.day>=self.max_day:
            print('Finished.')
            return True
        self.data = self.pct_dt.iloc[self.day:self.day+self.covWin,:]
        self.mu = self.get_mu()

        cov = self.data.cov()
        self.cov = np.block([[0,np.zeros(20)],[np.zeros([20,1]),cov]])

        #get the action for achive the SR maximum
        self.action = self.init_action
        self.action = self.optim().astype('float64')
        SR = self.SR(self.action)

        self.action_memory.append(self.action)
        print(f'Action:\n {self.action}')
        real_c = np.array(self.df.iloc[self.day+self.covWin,:]/self.df.iloc[self.day+self.covWin-1,:])
        print(f'Real return {real_c}')
        self.asset = self.p(self.action)*np.append(1+self.rf,real_c)
        self.asset_memory.append(self.asset)
        print(f'Asset:\n {self.asset}')
        #calculate the real asset
        self.weight = self.asset/self.asset.sum()
        self.day+=1
        top_3_idx = self.weight[1:].argsort()[-3:][::-1]
        top_3 = self.tick[top_3_idx]
        print(f'Day{self.day}, {self.date[self.day-1]}, SR {SR}, risk free {round(self.weight[0],2)} TOP3 {top_3},################ total {round(self.asset.sum(),2)}')
        print('#######################################################')
        return False
        
    def perf(self):
        while(1):
            done = self.step()
            if done:
                break
In [ ]:
lag = 0
df_pred = pd.read_csv("./prediction.csv", index_col=0).iloc[lag:,:]
df_real = df.loc['2019-11-15':,:].iloc[lag:,:]
model = portOptim(df_real,df_pred,dynamic_win = True)

Day0, initial portfolio [0.04761905 0.04761905 0.04761905 0.04761905 0.04761905 0.04761905
 0.04761905 0.04761905 0.04761905 0.04761905 0.04761905 0.04761905
 0.04761905 0.04761905 0.04761905 0.04761905 0.04761905 0.04761905
 0.04761905 0.04761905 0.04761905] , total 1.0
In [ ]:
model.perf()
In [ ]:
dynamic = pd.DataFrame(index = df_pred.index)
dynamic['value'] = np.array(model.asset_memory).sum(axis=1)-1
# cumReturn.to_csv('./dynamic_win.csv')
In [11]:
dynamic = pd.read_csv('./dynamic_win.csv').set_index('date')
# dynamic
In [13]:
NDX = pd.read_csv('./NDX_d_2012_2022.csv').set_index('date').loc['2019-12-31':,]
NDX_return = (NDX.adjcp.pct_change()+1).cumprod().dropna()-1
In [14]:
fixed = pd.read_csv('./fixed_window.csv')
In [17]:
from datetime import datetime as dt
import pyfolio
import matplotlib.pyplot as plt
import plotly
import plotly.graph_objs as go
In [18]:
time_ind = pd.Series(dynamic.index)
In [19]:
trace_dynamic = go.Scatter(x = time_ind, y = dynamic.value, mode = 'lines', name = 'max-SR(with dynamic window)')
trace_fixed = go.Scatter(x = time_ind, y = fixed.value, mode = 'lines', name = 'max-SR(with fixed windows)')
trace_baseline = go.Scatter(x = time_ind, y = NDX_return, mode = 'lines', name = 'NDX')
In [20]:
fig = go.Figure()
fig.add_trace(trace_dynamic)
fig.add_trace(trace_fixed)
fig.add_trace(trace_baseline)

fig.update_layout(
    legend=dict(
        x=0,
        y=1,
        traceorder="normal",
        font=dict(
            family="sans-serif",
            size=15,
            color="black"
        ),
        bgcolor="White",
        bordercolor="white",
        borderwidth=2
        
    ),
)
#fig.update_layout(legend_orientation="h")
fig.update_layout(title={
        #'text': "Cumulative Return using FinRL",
        'y':0.85,
        'x':0.5,
        'xanchor': 'center',
        'yanchor': 'top'})
#with Transaction cost
#fig.update_layout(title =  'Quarterly Trade Date')
fig.update_layout(
#    margin=dict(l=20, r=20, t=20, b=20),
    paper_bgcolor='rgba(1,1,0,0)',
    plot_bgcolor='rgba(1, 1, 0, 0)',
    #xaxis_title="Date",
    yaxis_title="Cumulative Return",
xaxis={'type': 'date', 
       'tick0': time_ind[0], 
        'tickmode': 'linear', 
       'dtick': 86400000.0 *80}

)

fig.update_xaxes(showline=True,linecolor='black',showgrid=True, gridwidth=1, gridcolor='LightSteelBlue',mirror=True)
fig.update_yaxes(showline=True,linecolor='black',showgrid=True, gridwidth=1, gridcolor='LightSteelBlue',mirror=True)
fig.update_yaxes(zeroline=True, zerolinewidth=1, zerolinecolor='LightSteelBlue')

fig.show()
In [21]:
baseline = np.append(1,np.array(NDX_return)+1)
baseline = (baseline[1:]/baseline[:-1]-1)
baseline = pd.DataFrame(baseline, index = pd.to_datetime(NDX_return.index),columns = ['return'])
In [22]:
dynamic_r = np.append(1,np.array(dynamic.value)+1)
dynamic_r = (dynamic_r[1:]/dynamic_r[:-1]-1).astype('float64')
dynamic_r = pd.DataFrame(dynamic_r,columns = ['return'])
dynamic_r.set_index(pd.to_datetime(dynamic.index),inplace = True)
In [23]:
fixed_r = np.append(1,np.array(fixed.value)+1)
fixed_r = (fixed_r[1:]/fixed_r[:-1]-1).astype('float64')
fixed_r = pd.DataFrame(fixed_r,columns = ['return'])
fixed_r.set_index(pd.to_datetime(dynamic.index),inplace = True)
In [22]:
with pyfolio.plotting.plotting_context(font_scale=1.1):
        pyfolio.create_full_tear_sheet(returns = dynamic_r['return'],
                                       benchmark_rets=baseline['return'], set_context=False)
Start date2020-01-02
End date2021-12-31
Total months24
Backtest
Annual return 64.454%
Cumulative returns 170.987%
Annual volatility 55.18%
Sharpe ratio 1.18
Calmar ratio 1.17
Stability 0.67
Max drawdown -55.226%
Omega ratio 1.24
Sortino ratio 1.78
Skew 0.14
Kurtosis 4.39
Tail ratio 1.03
Daily value at risk -6.694%
Alpha 0.20
Beta 1.32
Worst drawdown periods Net drawdown in % Peak date Valley date Recovery date Duration
0 55.23 2020-02-19 2020-03-18 2020-08-12 126
1 30.70 2021-01-25 2021-05-19 NaT NaN
2 17.31 2020-09-01 2020-09-23 2020-11-05 48
3 8.88 2020-02-04 2020-02-05 2020-02-18 11
4 7.73 2020-11-05 2020-11-20 2020-12-04 22
Stress Events mean min max
New Normal 0.26% -16.81% 18.81%
In [24]:
with pyfolio.plotting.plotting_context(font_scale=1.1):
        pyfolio.create_full_tear_sheet(returns = fixed_r['return'],
                                       benchmark_rets=baseline['return'], set_context=False)
Start date2020-01-02
End date2021-12-31
Total months24
Backtest
Annual return 61.514%
Cumulative returns 161.364%
Annual volatility 28.8%
Sharpe ratio 1.81
Calmar ratio 1.64
Stability 0.96
Max drawdown -37.439%
Omega ratio 1.41
Sortino ratio 2.69
Skew -0.10
Kurtosis 9.80
Tail ratio 1.00
Daily value at risk -3.422%
Alpha 0.24
Beta 0.87
Worst drawdown periods Net drawdown in % Peak date Valley date Recovery date Duration
0 37.44 2020-02-19 2020-03-20 2020-05-27 71
1 9.52 2020-09-02 2020-09-21 2020-10-22 37
2 7.10 2021-02-16 2021-03-08 2021-03-16 21
3 6.00 2021-12-10 2021-12-20 2021-12-29 14
4 5.67 2021-05-07 2021-05-12 2021-05-28 16
Stress Events mean min max
New Normal 0.21% -12.10% 12.20%